﻿using AngleSharp.Html.Parser;
using DotnetSpider.Core;
using DotnetSpider.Core.Scheduler;
using SpiderPlatform.Pipe;
using SpiderPlatform.Processor;
using System;
using System.Collections.Generic;
using System.Net;
using System.Text;

namespace SpiderPlatform.Service
{
    public class OscartService
    {
        public void doWork()
        {
            Logger.Info("获取总页数...");
            var pageInfo = "https://www.oscart.com/products/?";
            WebClient client = new WebClient();
            var pageContent=client.DownloadString(pageInfo);
            HtmlParser htmlParser = new HtmlParser();
            var dom=htmlParser.ParseDocument(pageContent);
            int pageTotal = 60;          
            var page = dom.QuerySelector("div.ui.menu.nav-actions.borderless div.item");
            if (page != null)
            {
                var pStr = page.InnerHtml.Replace(" 件商品", "");
                pageTotal = int.Parse(pStr) / 16 + 1;
                Logger.Info("总页数===》" + pageTotal);
            }
            var baseUrl = "https://www.oscart.com/products/index.html+infinite?page=";
            List<string> list = new List<string>();
            for (int i = 1; i <= pageTotal; i++)
            {                
                list.Add(baseUrl + i.ToString());
            }
            var site = new Site
            {

                CycleRetryTimes = 3,
               // SleepTime = 300,
                Headers = new Dictionary<string, string>()                {

                    { "Accept","text/html, */*; q=0.01" },
                    { "Referer", "https://www.oscart.com"},
                    { "Cache-Control","no-cache" },
                    { "Connection","keep-alive" },
                    { "Content-Type","application/x-www-form-urlencoded; charset=UTF-8" },
                    { "User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36"}
                }
            };
            List<Request> resList = new List<Request>();
            foreach (var item in list)
            {
                Request res = new Request();
                res.Url = item;
                res.Method = System.Net.Http.HttpMethod.Get;
                resList.Add(res);
            }

            var spider = DotnetSpider.Core.Spider.Create(site, new QueueDuplicateRemovedScheduler(), new OscartProcessor())
           .AddStartRequests(resList.ToArray())
           .AddPipeline(new OscartPipe());           
            spider.ThreadNum = 5;
            spider.Run();
            Logger.Info("任务完成");
            Console.ReadKey();
        }
    }
}
